package com.alex.utils.textsearch;

import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.LineIterator;

import com.alex.utils.common.SecurityUtils;
import com.alex.utils.common.TrustAllCertificates;
import com.alex.utils.http.HttpClientFactory;
import com.alex.utils.http.HttpResult;
import com.alex.utils.http.HttpUtils;
import com.alex.utils.http.RequestBuilderFactory;
import com.alex.utils.http.cache.FileHttpCacheProvider;
import com.alex.utils.http.cache.HttpCacheProvider;
import com.alex.utils.textsearch.filter.PairParenthesesFilter;
import com.alex.utils.textsearch.filter.RussianTextFilter;
import com.alex.utils.textsearch.processor.MakeSingleWhitespacesProcessor;
import com.alex.utils.textsearch.processor.ReplaceQuotesProcessor;

public class TextSearch {
	
	private static final Pattern META_CHARSET_PATTERN = Pattern.compile("<meta.*?charset=([^\"'>; ]+)", Pattern.CASE_INSENSITIVE);

	private RequestBuilderFactory cachingRequestBuilderFactory;

	private HttpCacheProvider cacheProvider;

	private LineIterator iterator;

	private String projectDir = "/home/alex/Desktop/keywords/раскрутка_сайта/";

	private AtomicInteger count = new AtomicInteger(0);

	public static void main(String[] args) throws Exception {
		new TextSearch().search();
	}

	private void search() throws Exception {
		TrustAllCertificates.enable();
		
		cacheProvider = new FileHttpCacheProvider(new File(System.getProperty("user.home") + "/.text-searcher-cache"));
		HttpClientFactory httpClientFactory = new HttpClientFactory();
		httpClientFactory.setThreadSafe(true);
		cachingRequestBuilderFactory = new RequestBuilderFactory(httpClientFactory.create());
		cachingRequestBuilderFactory.setCacheProvider(cacheProvider);

		iterator = FileUtils.lineIterator(new File(projectDir + "site_urls.txt"));

		Thread[] threads = new Thread[5];
		for (int i = 0; i < threads.length; i++) {
			threads[i] = new Worker();
			threads[i].start();
		}

		for (Thread thread : threads) {
			thread.join();
		}
	}

	private synchronized String getNextURL() {
		if (iterator.hasNext()) {
			return iterator.next();
		} else {
			return null;
		}
	}

	class Worker extends Thread {

		private HtmlTextExtractor htmlTextExtractor = new HtmlTextExtractor();
		
		public Worker() {
			Collection<SentenceFilter> sentenceFilters = new ArrayList<SentenceFilter>();
			sentenceFilters.add(new RussianTextFilter());
			sentenceFilters.add(new PairParenthesesFilter());
			
			Collection<SentenceProcessor> sentenceProcessors = new ArrayList<SentenceProcessor>();
			sentenceProcessors.add(new MakeSingleWhitespacesProcessor());
			sentenceProcessors.add(new ReplaceQuotesProcessor());
			
			htmlTextExtractor.setSentenceProcessors(sentenceProcessors);
			htmlTextExtractor.setSentenceFilters(sentenceFilters);
		}

		@Override
		public void run() {
			while (true) {
				try {
					String url = getNextURL();
					if (url == null) {
						return;
					}

					String html = downloadSite(url);
					String text = htmlTextExtractor.extract(html);

					if (!text.isEmpty()) {
						String name = SecurityUtils.encodeSHA1(url);
						File file = new File(projectDir + "texts/" + name.charAt(0) + "/" + name + ".txt");
						// FileUtils.write(file, url + "\n\n" + text);
						FileUtils.write(file, text);
					}

					int c = count.incrementAndGet();
					if (c % 100 == 0) {
						System.out.println(c);
					}
				} catch (Exception e) {
					e.printStackTrace();
				}
			}
		}

		private String downloadSite(String url) throws IOException {
			HttpResult httpResult = cachingRequestBuilderFactory.get(url).getResult();
			byte[] data = httpResult.getResponseBodyAsByteArray();
			
			String html = null;
			Charset charset8859_1 = Charset.forName("iso-8859-1");
			
			String charsetName = HttpUtils.getResponseCharset(httpResult.getResponse());
						
			if (charsetName == null || !Charset.isSupported(charsetName)) {
				html = new String(data, charset8859_1);
				charsetName = determineCharsetFromHtml(html);
				
				if (charsetName != null && Charset.isSupported(charsetName)) {
					Charset charset = Charset.forName(charsetName);
					if (!charset8859_1.equals(charset)) {
						html = new String(data, charset);
					}
				}
			} else {
				html = new String(data, charsetName);
			}
			
			return html;
		}

		private String determineCharsetFromHtml(String html) {
			Matcher matcher = META_CHARSET_PATTERN.matcher(html);
			if (matcher.find()) {
				return matcher.group(1);
			} else {
				return null;
			}
		}
	}
}
